#! /bin/bash
# set -ex

export ONEFLOW_GPT_SEQ_LENGTH=1024
export ONEFLOW_GPT_NUM_LAYERS=12
export ONEFLOW_GPT_HIDDEN_SIZE=768
export ONEFLOW_GPT_NUM_ATTENTION_HEADS=12
export ONEFLOW_GPT_NUM_GPUS_PER_NODE=1
export ONEFLOW_GPT_MICRO_BATCH_SIZE=8

export PYTHONUNBUFFERED=1
export NCCL_DEBUG=INFO
export ONEFLOW_DEBUG_MODE=1
export ONEFLOW_PROFILER_KERNEL_PROFILE_KERNEL_FORWARD_RANGE=1
export ONEFLOW_GTP_PROFILE_FILE="117M_1n1d_bz8"

source $(dirname $0)/pretrain.sh
